phase1, phase2: implement force scheduler logic
authorJo-Philipp Wich <[email protected]>
Mon, 3 Feb 2020 15:50:06 +0000 (16:50 +0100)
committerJo-Philipp Wich <[email protected]>
Mon, 3 Feb 2020 15:50:06 +0000 (16:50 +0100)
Signed-off-by: Jo-Philipp Wich <[email protected]>
phase1/master.cfg
phase2/master.cfg

index 24262aded8f7e6da0a2711d84af059a1ccdb890c..83d9e8b1f829b59e6d6048cfcceb4d96981f7ca3 100644 (file)
@@ -205,17 +205,107 @@ c['change_source'].append(GitPoller(
 
 from buildbot.schedulers.basic import SingleBranchScheduler
 from buildbot.schedulers.forcesched import ForceScheduler
+from buildbot.schedulers.forcesched import BaseParameter
+from buildbot.schedulers.forcesched import ValidationError
+from buildbot.plugins import schedulers
+from buildbot.plugins import util
 from buildbot.changes import filter
+
+class TagChoiceParameter(BaseParameter):
+       spec_attributes = ["strict", "choices"]
+       type = "list"
+       strict = True
+
+       def __init__(self, name, label=None, **kw):
+               super().__init__(name, label, **kw)
+               self._choice_list = []
+
+       @property
+       def choices(self):
+               taglist = []
+               basever = re.search(r'-([0-9]+\.[0-9]+)$', repo_branch)
+
+               if basever:
+                       findtags = subprocess.Popen(
+                               ['git', 'ls-remote', '--tags', repo_url],
+                               stdout = subprocess.PIPE)
+
+                       while True:
+                               line = findtags.stdout.readline()
+
+                               if not line:
+                                       break
+
+                               tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())
+
+                               if tagver and tagver[1].find(basever[1]) == 0:
+                                       taglist.append(tagver[1])
+
+               taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
+               taglist.insert(0, '')
+
+               self._choice_list = taglist
+
+               return self._choice_list
+
+       def parse_from_arg(self, s):
+               if self.strict and s not in self._choice_list:
+                       raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
+               return s
+
 c['schedulers'] = []
 c['schedulers'].append(SingleBranchScheduler(
-       name="all",
-       change_filter=filter.ChangeFilter(branch=repo_branch),
-       treeStableTimer=60,
-       builderNames=targets))
+       name            = "all",
+       change_filter   = filter.ChangeFilter(branch=repo_branch),
+       treeStableTimer = 60,
+       builderNames    = targets))
 
 c['schedulers'].append(ForceScheduler(
-       name="force",
-       builderNames=targets))
+       name         = "force",
+       buttonName   = "Force builds",
+       label        = "Force build details",
+       builderNames = [ "00_force_build" ],
+
+       codebases = [
+               util.CodebaseParameter(
+                       "",
+                       label      = "Repository",
+                       branch     = util.FixedParameter(name = "branch",     default = ""),
+                       revision   = util.FixedParameter(name = "revision",   default = ""),
+                       repository = util.FixedParameter(name = "repository", default = ""),
+                       project    = util.FixedParameter(name = "project",    default = "")
+               )
+       ],
+
+       reason = util.StringParameter(
+               name     = "reason",
+               label    = "Reason",
+               default  = "Trigger build",
+               required = True,
+               size     = 80
+       ),
+
+       properties = [
+               util.NestedParameter(
+                       name="options",
+                       label="Build Options",
+                       layout="vertical",
+                       fields=[
+                               util.ChoiceStringParameter(
+                                       name    = "target",
+                                       label   = "Build target",
+                                       default = "all",
+                                       choices = [ "all" ] + targets
+                               ),
+                               TagChoiceParameter(
+                                       name    = "tag",
+                                       label   = "Build tag",
+                                       default = ""
+                               )
+                       ]
+               )
+       ]
+))
 
 ####### BUILDERS
 
@@ -231,8 +321,11 @@ from buildbot.steps.transfer import FileUpload
 from buildbot.steps.transfer import FileDownload
 from buildbot.steps.transfer import StringDownload
 from buildbot.steps.master import MasterShellCommand
+from buildbot.plugins import steps
 from buildbot.process.properties import Interpolate
+from buildbot.process.properties import Property
 from buildbot.process import properties
+from buildbot.config import BuilderConfig
 
 
 CleanTargetMap = [
@@ -283,7 +376,7 @@ def IsGitCleanRequested(step):
 
 def IsTaggingRequested(step):
        val = step.getProperty("tag")
-       if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
+       if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
                return True
        else:
                return False
@@ -295,7 +388,7 @@ def IsNoMasterBuild(step):
        return repo_branch != "master"
 
 def GetBaseVersion():
-       if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
+       if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
                return repo_branch.split('-')[1]
        else:
                return "master"
@@ -303,7 +396,7 @@ def GetBaseVersion():
 @properties.renderer
 def GetVersionPrefix(props):
        basever = GetBaseVersion()
-       if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
+       if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
                return "%s/" % props["tag"]
        elif basever != "master":
                return "%s-SNAPSHOT/" % basever
@@ -390,6 +483,29 @@ def NetLockUl(props):
        else:
                return []
 
+def TagPropertyValue(props):
+       if props.hasProperty("options"):
+               options = props.getProperty("options")
+               if type(options) is dict:
+                       return options.get("tag")
+       return None
+
+def IsTargetSelected(target):
+       def CheckTargetProperty(step):
+               try:
+                       options = step.getProperty("options")
+                       if type(options) is dict:
+                               selected_target = options.get("target", "all")
+                               if selected_target != "all" and selected_target != target:
+                                       return False
+               except KeyError:
+                       pass
+
+               return True
+
+       return CheckTargetProperty
+
 def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
        try:
                seckey = base64.b64decode(seckey)
@@ -462,6 +578,13 @@ slaveNames = [ ]
 for slave in c['workers']:
        slaveNames.append(slave.workername)
 
+force_factory = BuildFactory()
+
+c['builders'].append(BuilderConfig(
+       name        = "00_force_build",
+       workernames = slaveNames,
+       factory     = force_factory))
+
 for target in targets:
        ts = target.split('/')
 
@@ -1167,10 +1290,17 @@ for target in targets:
                alwaysRun = True,
        ))
 
-       from buildbot.config import BuilderConfig
-
        c['builders'].append(BuilderConfig(name=target, workernames=slaveNames, factory=factory, nextBuild=GetNextBuild))
 
+       c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
+       force_factory.addStep(steps.Trigger(
+               name = "trigger_%s" % target,
+               description = "Triggering %s build" % target,
+               schedulerNames = [ "trigger_%s" % target ],
+               set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
+               doStepIf = IsTargetSelected(target)
+       ))
+
 
 ####### STATUS TARGETS
 
index f5be23ed31bfeb247eb18c9e632033dd63d54975..d3a76144b33ff31cc3170de72e01c7d8ec72fe14 100644 (file)
@@ -195,17 +195,59 @@ def branch_change_filter(change):
 
 from buildbot.schedulers.basic import SingleBranchScheduler
 from buildbot.schedulers.forcesched import ForceScheduler
+from buildbot.plugins import schedulers
+from buildbot.plugins import util
+from buildbot.plugins import steps
 from buildbot.changes import filter
+
 c['schedulers'] = []
 c['schedulers'].append(SingleBranchScheduler(
-       name="all",
-       change_filter=filter.ChangeFilter(filter_fn=branch_change_filter),
-       treeStableTimer=60,
-       builderNames=archnames))
+       name            = "all",
+       change_filter   = filter.ChangeFilter(filter_fn=branch_change_filter),
+       treeStableTimer = 60,
+       builderNames    = archnames))
 
 c['schedulers'].append(ForceScheduler(
-       name="force",
-       builderNames=archnames))
+       name         = "force",
+       buttonName   = "Force builds",
+       label        = "Force build details",
+       builderNames = [ "00_force_build" ],
+
+       codebases = [
+               util.CodebaseParameter(
+                       "",
+                       label      = "Repository",
+                       branch     = util.FixedParameter(name = "branch",     default = ""),
+                       revision   = util.FixedParameter(name = "revision",   default = ""),
+                       repository = util.FixedParameter(name = "repository", default = ""),
+                       project    = util.FixedParameter(name = "project",    default = "")
+               )
+       ],
+
+       reason = util.StringParameter(
+               name     = "reason",
+               label    = "Reason",
+               default  = "Trigger build",
+               required = True,
+               size     = 80
+       ),
+
+       properties = [
+               util.NestedParameter(
+                       name="options",
+                       label="Build Options",
+                       layout="vertical",
+                       fields=[
+                               util.ChoiceStringParameter(
+                                       name    = "architecture",
+                                       label   = "Build architecture",
+                                       default = "all",
+                                       choices = [ "all" ] + archnames
+                               )
+                       ]
+               )
+       ]
+))
 
 ####### BUILDERS
 
@@ -221,10 +263,12 @@ from buildbot.steps.transfer import FileDownload
 from buildbot.steps.transfer import StringDownload
 from buildbot.steps.master import MasterShellCommand
 from buildbot.process.properties import WithProperties
+from buildbot.process.properties import Property
+from buildbot.config import BuilderConfig
 
 
 def GetDirectorySuffix(props):
-       verpat = re.compile('^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$')
+       verpat = re.compile(r'^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$')
        if props.hasProperty("release_version"):
                m = verpat.match(props["release_version"])
                if m is not None:
@@ -245,6 +289,21 @@ def GetCwd(props):
        else:
                return "/"
 
+def IsArchitectureSelected(target):
+       def CheckArchitectureProperty(step):
+               try:
+                       options = step.getProperty("options")
+                       if type(options) is dict:
+                               selected_arch = options.get("architecture", "all")
+                               if selected_arch != "all" and selected_arch != target:
+                                       return False
+               except KeyError:
+                       pass
+
+               return True
+
+       return CheckArchitectureProperty
+
 def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
        try:
                seckey = base64.b64decode(seckey)
@@ -267,6 +326,13 @@ slaveNames = [ ]
 for slave in c['workers']:
        slaveNames.append(slave.workername)
 
+force_factory = BuildFactory()
+
+c['builders'].append(BuilderConfig(
+       name        = "00_force_build",
+       workernames = slaveNames,
+       factory     = force_factory))
+
 for arch in arches:
        ts = arch[1].split('/')
 
@@ -604,10 +670,16 @@ for arch in arches:
                alwaysRun = True
        ))
 
-       from buildbot.config import BuilderConfig
-
        c['builders'].append(BuilderConfig(name=arch[0], workernames=slaveNames, factory=factory))
 
+       c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % arch[0], builderNames=[ arch[0] ]))
+       force_factory.addStep(steps.Trigger(
+               name = "trigger_%s" % arch[0],
+               description = "Triggering %s build" % arch[0],
+               schedulerNames = [ "trigger_%s" % arch[0] ],
+               set_properties = { "reason": Property("reason") },
+               doStepIf = IsArchitectureSelected(arch[0])
+       ))
 
 ####### STATUS arches